########################################################################
#  Pyfm - personal finance data-base
#  Copyright (C) 2007,8 Ehud Ben-Reuven
#  udi@benreuven.com
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
########################################################################
"""Report functions
====================
report tools on the data-base
"""
from obj.obj import db
from obj.dt import DT
from obj.flat import stagvalue,add2line,sobj,sdict,squote
from obj.objtools import get_filter_tags,get_command_tags
from model import *
from modellib import *
from trnlib import *
from tagsyntax import *
from lib.quoteinterface import getFXrate
import copy
from lib import conf
from gainloss import *
try:
    import lib.xls
except ImportError:
    print "Warning: lib.xls not found. Importing from Excel won't work."
    pass

def do_rp(self,line={}):
    """rp [CURDEF=<currency>] [DTASOF=<report-date>] [Holding=<holding-filter>] [Statement=<statement-filter>]
Display latest UNITS and UNITPRICE of holdings (up until DTASOF) and MKTVAL.
For example:
rp Holding=|ACCTGRP=B| Statement=|FILE=_MERGED_|
"""
    header=[]
    table=[]

    curdef=getcurdef(line)
    dtre=line.pop('DTASOF',[DTASOF('NOW')])[0]
    # Check if there are tags that should be printed
    filter_tags=get_filter_tags(line)
    command_tags=get_command_tags(line,filter_tags)

    statement_filter=getstatement_filter(line,dtre)
    totalmktval=GainLoss() # Currency is irrelevant, we want percentage
    for h in getholdings(line):
        statement=getstatement(statement_filter,h)
        if not statement: continue
        print squote(str(h)),statement['DTASOF'][0],statement['UNITS'][0],
        header=['Holding','DTASOF (UNITS)','UNITS']
        row=[h,statement['DTASOF'][0],statement['UNITS'][0]]

        dtprice,unitprice,cur=gethistory(h,dtre)        
        checking = unitprice==1 and h.ischecking()
        header+=['UNITPRICE']
        if not checking:
            print unitprice,
            row.append(unitprice)
        else:
            row.append(None)
        if cur!=curdef:
            print cur,
        else:
            pass        

        if unitprice and not statement.MKTVAL:
            mktval=MKTVAL(statement.UNITS[0]*unitprice)
        elif statement.MKTVAL:
            mktval=statement.MKTVAL[0]
        else:
            mktval=None
        if mktval:
            if cur and cur!=curdef:
                mktval=MKTVAL(mktval*getFXrate(dtre,cur,curdef))
        header+=['MKTVAL']
        if mktval:
            print str(mktval)+scurdef(curdef,cur),
            row+=[mktval]
        else:
            row+=[0.]
        header+=['DTASOF (UNITPRICE)']
        if not checking and dtprice and dtprice!=statement['DTASOF'][0]:
            print dtprice,
            row+=[dtprice]
        else:
            row+=[None]
        components=getcomposition(h,dtre)
        if mktval:
            for c in components:
                totalmktval.add(c,components[c]*float(mktval))
        if 'PRINT' in command_tags:
            for tag in command_tags['PRINT']:
                if tag=='COMPONENT':
                    values=components
                else:
                    values=h.rget(tag)
                print stagvalue(tag,values,short=True),
                header+=[tag]
                row+=[values]
        print
        table.append(row)

    holding_filter=getfilter(line,'Holding')
    for tag in holding_filter:
        print stagvalue(tag,holding_filter[tag]),
    sum_mktval=sum(sum(x) for x in totalmktval.values())/100.
    print stagvalue('DTASOF',dtre),stagvalue('MKTVAL',sum_mktval)
    for c in totalmktval:
        totalmktval[c][0] /= sum_mktval
        totalmktval[c][1] /= sum_mktval
    totalmktval.pprint()
    if '-X' in line:
        lib.xls.table(header,table,sumlist=['MKTVAL'])


def do_rug(self,line={}):
    """Report Unrealized Gain
rug DTASOF=NOW Holding={} Transaction=|FILE=_MERGED_| CURDEF=USD*
break the gain into short (less than a year) or long
sub-break into gain and loss."""
    curdef=getcurdef(line)
    dtasof=line.get('DTASOF',[DTASOF('NOW')])[0]
    transaction_filter=gettransaction_filter(line)
    totalgainloss=GainLoss(curdef)
    for h in getholdings(line):
        if h.ischecking(): continue
        trans=gettransactions(transaction_filter,h)
        if not trans: continue
        trans.sort(key=lambda t: t.dttrade()) # oldest first
        units=[] # TODO instead of building these 3 arrays, use an list of Transaction
        price=[]
        date=[]
        trntypelist=[]
        curlist=[]
        currency=h.Asset[0].CURRENCY
        if currency: currency=currency[0]
        sumunits=0.
        for t in trans:
            dt=t.DTSETTLE[0]
            if dt>dtasof: break
            if not (t.UNITS and t.UNITS[0]): continue
            u=t.UNITS[0]
            sumunits+=u
            if u<0: continue # Skip sell transactions
            units.append(t.UNITS[0])
            p,cur=getunitpricecur(t)
            if not p:
                price_dt,p,cur=getprice(h,dt) # This can happen on a transfer TODO FIX THIS
                if not p: p=0
            if cur:
                if not currency: currency=cur
            if not cur:
                cur = currency if currency else curdef
            price.append(p)
            date.append(dt)
            trntypelist.append(t.TRNTYPE[0])
            curlist.append(cur)
        if sumunits<1e-6: continue
        dtpriceasof,priceasof,cur=getprice(h,dtasof)
        gainloss=GainLoss(curdef)
        for u,p,dt,tt,c in reversed(zip(units,price,date,trntypelist,curlist)):
            if sumunits<=1e-6: break
            buyunits=min(sumunits,u)
            sumunits-=buyunits
            gaintype='Long' if dtasof-dt>365 else 'Short'
            if priceasof*buyunits*getFXrate(dtasof,cur,curdef)-p*buyunits*getFXrate(dt,c,curdef) > 0:
                gaintype+='.Gain'
            else:
                gaintype+='.Lose'
            gainloss.add('Unrealized.'+gaintype+'.Unrealized',priceasof*buyunits,dtasof,cur)
            gainloss.add('Unrealized.'+gaintype+'.'+tt,-p*buyunits,dt,c)
        totalgainloss.join(gainloss)
        if sumunits>1e-6: perr('Units without purchase',h)
        print h,dtasof,gainloss
    totalgainloss.pprint()

def do_rrg(self,line={}):
    """Report Realized Gain
rrg DTEND=NOW DTSTART=PAST Holding={} Transaction=|FILE=_MERGED_| CURDEF=USD*
break capital gain into short (less than a year) or long
and collect interest, dividend and others.
call f Class=Transaction FIX=TRUE ... before calling this.
"""
    curdef=getcurdef(line)
    dtstart,dtend=popdtstartend(line)

    transaction_filter=gettransaction_filter(line)
    totalgainloss=GainLoss(curdef)
    for h in getholdings(line):
        trans=gettransactions(transaction_filter,h)
        if not trans: continue
        trans.sort(key=lambda t: t.dttrade()) #oldest first.
        currency=h.Asset[0].CURRENCY
        if currency: currency=currency[0]
        buy=[]
        gainloss=GainLoss(curdef)
        for t in trans:
            dt=t.DTSETTLE[0]
            if dt>dtend: break
            p,cur=getunitpricecur(t)
            if cur:
                if not currency: currency=cur
            if not cur:
                cur = currency if currency else curdef

            trntype = t.TRNTYPE[0]
            if trntype in ['TRANSFER','IN','OUT']: #FIXME handle transfer between holdings/accounts
                perr('Not taking TRANSFER/IN/OUT',[t])
                continue
            
            if h.ischecking():
                if trntype in ['INT','DIV'] and dt>=dtstart and t.TOTAL and t.TOTAL[0]:
                    gainloss.add('Interest.'+trntype,t.TOTAL[0],dt,cur)                    
                continue

            u=t.UNITS[0] if t.UNITS else 0.
            if u>0:
                if p==None:
                    perr('No price on buy',t)
                    continue
                # Push purchase into FIFO stack
                buy.append([u,p,dt,trntype])
                # ACCRDINT
                if dt>=dtstart and t.INTEREST and t.INTEREST[0]:
                    gainloss.add('Interest.'+trntype+'.accrdint',t.INTEREST[0],dt,cur)
            if u<0:
                if p==None:
                    perr('No price on sell',t)
                    continue
                u=t.UNITS[0]
                # Pull matching purchase from FIFO
                for b in buy:
                    bunits=min(b[0],-u)
                    if dt>=dtstart:
                        gaintype='Long' if dt-b[2]>365 else 'Short'
                        if bunits*p-bunits*b[1]>0:
                            gaintype+='.Gain'
                        else:
                            gaintype+='.Lose'
                        gainloss.add('Purchase.'+gaintype+'.'+b[3],-bunits*b[1],b[2],cur)
                        gainloss.add('Purchase.'+gaintype+'.'+trntype,bunits*p,dt,cur)
                    u+=bunits
                    b[0]-=bunits
                    if u>= 0: break
                if u<0: perr('Unballanced sell',t)
            if dt>=dtstart and t.INTEREST and t.INTEREST[0]:
                gainloss.add('Interest.'+trntype,t.INTEREST[0],dt,cur)
        if gainloss:
            totalgainloss.join(gainloss)
            print h,gainloss
    totalgainloss.pprint()

def do_rs(self,line={}):
    """rs [Holding=<holding-filter>] [Statement=<statement-filter>] [DTEND=<all-dates>]
For each holding (that fits the holding-filter), report all statements (that fit the statement-filter)
"""
    statement_filter=getstatement_filter(line,
                                         dtre=line.get('DTEND',[None])[0],
                                         default={})
    sum=0.
    for h in getholdings(line):
        add2line(statement_filter,'!DTASOF','SORT')
        for hs in statement_filter.setdefault('HoldingSource',[{}]):
            hs['Holding']=[h]

        mstatements=db.getallobjects(Statement,statement_filter)
        if not mstatements: continue
        print h,
        if len(mstatements)>1: print
        for mstatement in mstatements:
            print mstatement.DTASOF[0],mstatement.UNITS[0],mstatement.HoldingSource[0].FILE[0]

def do_cs(self,line):
    """cs [Holding={<holding-filter>}] [Statement={<statement-filter>}] [RefStatement={<ref-statement-filter>}]
For every holding, compare each statement (matching statement-filter) with the latest statement found by
ref-statement-filter that is at or before the compared statement.
If RefStatement is not specified then the ref-statement-filter is made to be identical to statement-filter except for the fact that
HoldingSource={FILE=_MERGED_}
This is a way to compare if the merged transaction information summed up to merged statements to other statement information.
Note: you need to call ms command before comparing merged statements

For example:
cs Holding=|ACCTGRP=B| Statement={DTASOF!MAX}
Will check if the latest Statement for all holding in the accounts of group B are identical to the merged statements.
"""
    statement_filter=getstatement_filter(line,
                                         dtre=line.get('DTEND',[None])[0],
                                         default={'HoldingSource':[{'!=FILE':['_MERGED_']}]})
    add2line(statement_filter,'!DTASOF','SORT')

    ref_statement_filter=getfilter(line,'RefStatement')
    if not ref_statement_filter:
        ref_statement_filter=line_copy(statement_filter) # deep because we change the inside of values
        for tag,raw_tag,op in get_filter_tags(ref_statement_filter):
            if raw_tag=='DTASOF': del ref_statement_filter[tag]
        for hs in ref_statement_filter.setdefault('HoldingSource',[{}]):
            for tag,raw_tag,op in get_filter_tags(hs):
                if raw_tag=='FILE': del hs[tag]            
            hs['FILE']=['_MERGED_']
    # This is critical for the working of the test
    add2line(ref_statement_filter,'!DTASOF','SORT')

    for h in getholdings(line):
        statements=getstatements(statement_filter,h)
        if not statements: continue

        ref_statements=getstatements(ref_statement_filter,h)
        # We want to find the latest ref-statement first
        ref_statements.reverse()

        statements=[s for s in statements if s not in ref_statements]
        if not statements:
            if not all(s.UNITS[0]==0 for s in ref_statements): # avoid problems with UNITS=None
                print sobj(h),'no statements','vs.','reference statements'
            continue

        lasts=lastr=None
        hline={}
        h.build_line(hline)

        for s in statements:            
            r=None
            nextr=None
            for rr in ref_statements:
                if rr.DTASOF[0]<=s.DTASOF[0]:
                    r=rr
                    break
                nextr=rr
            #if r==lastr: r=None

            # Build a line that will help the user in locating which transactions may be responsible
            # to any differences in statement value
            sline=hline.copy()
            if 'DTEND' in line:
                sline['DTSETTLE']=s.DTASOF[0]
            else:
                # If DTEND is not specified we have a full list of statements to report about and
                # if there is a problem in the current statement then its cause is somewhere before
                # the current statement but after the previouse statement which we already reported about.
                sline['<DTSETTLE']=[DT(s.DTASOF[0].str6())+1]
                if lasts:
                    sline['>=DTSETTLE']=[DT(lasts.DTASOF[0].str6())]

            ok=False
            if not r:
                if s.UNITS[0] and (not lasts or abs(s.UNITS[0]-lasts.UNITS[0])>0.5):
                    print s,'vs.','no reference','DIFF='+str(s.UNITS[0]),'Transaction='+sdict(sline,'||'),
                else:
                    ok=True
            elif r==lastr:
                if s.UNITS[0] and (not lasts or abs(s.UNITS[0]-lasts.UNITS[0])>0.5):
                    print s,'vs.','no new ref','DIFF='+str(s.UNITS[0]-r.UNITS[0]),'Transaction='+sdict(sline,'||'),
                else:
                    ok=True
            elif abs(s.UNITS[0] - r.UNITS[0])>0.5:
                print s,'vs.',r,'DIFF='+str(s.UNITS[0] - r.UNITS[0]),'Transaction='+sdict(sline,'||'),
            else:
                ok=True
            # If we have a problem then perhaps the next reference statement is OK and the problem is not bad after all
            if not ok:
                if nextr and abs(s.UNITS[0] - nextr.UNITS[0])<=0.5:
                    print 'corrected on',nextr
                else:
                    print
            lasts=s
            if r: lastr=r


def do_cd(self,line):
    """cd [Holding={<holding-filter>}] [Asset={<asset-filter>}]
Find different holdings with same  asset"""
    asset_filter=line.pop('Asset',[{}])
    if len(asset_filter)!=1: raise Exception('Cant handle more than one Asset filter')
    asset_filter=asset_filter[0]
    assets=db.getallobjects(Asset,asset_filter)

    holding_filter=line.pop('Holding',[{}])
    if len(holding_filter)!=1: raise Exception('Cant handle more than one Holding filter')
    holding_filter=holding_filter[0]

    holdings=db.getallobjects(Holding,holding_filter)

    for a in assets:
        hs=[h for h in holdings if any(ha==a for ha in h.Asset)]
        if len(hs)>1:
            for h  in hs:
                print h,
            print

def do_rt(self,line={}):
    """rt [CURDEF=<currency>] [Holding=<holding-filter>] [Transaction=<transaction-filter>] [DTSTART=<dtstart>] [DTEND=<dtend>]
Display transactions of holdings. If transaction filter is not given then only _MERGED_ transactions will be taken:
rt Holding={HIDX=16} Transaction=:FILE=_MERGED_:
You can limit the settlement dates of transactions to be between DTSTART and DTEND.
So:
rt Holding=:HIDX=16: DTSTART=1-Jan-06 DTEND=1-Jan-07
is the same as
rt Holding={HIDX=16} Transaction={DTSETTLE>1-Jan-06 DTSETTLE<1-Jan-07 Transaction=:FILE=_MERGED_:}
"""
    curdef=getcurdef(line)
    transaction_filter=gettransaction_filter(line)

    command_tags=get_command_tags(line)

    for h in getholdings(line):
        mtrans=gettransactions(transaction_filter,h)
        if not mtrans: continue
        print `h`
        for mtran in mtrans:
            if not trntotal(mtran): continue
            trnprint(mtran,curdef)
            if 'PRINT' in command_tags:
                for tag in command_tags['PRINT']:
                    print stagvalue(tag,mtran[tag]),
            print
